# !/usr/bin/env python
# -*- coding: utf-8 -*-
import sys
import os
import re
from public import  generate_random_ua, get_chrome_version, is_floatstr,has_ip_address
import threading
from bs4 import BeautifulSoup
from sqlalchemy.orm import sessionmaker
# 创建会话
from PyQt5.QtGui import QStandardItemModel
from PyQt5.QtWidgets import QFileDialog
import pymssql
import openpyxl
from public import gettablename
from public import dbhelper
from public import getbrandinfo, getaccountinfo, write_log
from datetime import datetime
from PyQt5.QtGui import QTextCursor
import datetime
import random
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from PyQt5.QtWidgets import QApplication, QTabWidget, QMessageBox, QMenu, QMenuBar, QAction

from PyQt5.QtWidgets import (QApplication, QWidget, QComboBox, QLineEdit,
                             QPushButton, QTableView, QTextEdit, QVBoxLayout,
                             QLabel, QHBoxLayout, QMessageBox, QProgressBar,
                             QInputDialog)
from PyQt5.QtCore import QAbstractTableModel, Qt
from PyQt5.QtWidgets import QDesktopWidget
from PyQt5.QtWidgets import QWidget, QVBoxLayout, QTableView
from PyQt5.QtSql import QSqlDatabase, QSqlQuery, QSqlTableModel
from PyQt5.QtCore import Qt, QThread
from public import DatabaseModel
from PyQt5.QtWidgets import QApplication, QVBoxLayout, QWidget, QTableView, QTextEdit, QScrollBar
from public import (get_chrome_version, getbrandinfo, getconnection_url)
import os
from PyQt5.QtWidgets import QApplication, QWidget, QLabel
from PyQt5.QtCore import pyqtSignal, QObject
from MySQLdb import connect
import numpy as np
import pyodbc
from sqlalchemy.sql import text
import openpyxl
import pymysql
from sqlalchemy.engine.url import URL
import locale
import sys, time, random
import re
import sys
import logging
from openpyxl import load_workbook
from sqlalchemy import create_engine, false, table
import requests
import pandas
from selenium.webdriver.common.keys import Keys
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from lxml import etree
import pandas as pd
from selenium.webdriver.chrome.service import Service
import socket
from PyQt5.QtWidgets import QInputDialog
from public import is_floatstr
from trio import current_time
from sqlalchemy.orm import sessionmaker
import shutil
import copy

from selenium.webdriver import ActionChains
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from subprocess import CREATE_NO_WINDOW
import traceback
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
from sqlalchemy import true
from public import (getbrandinfo, getaccountinfo)
from public import user_agents, generate_random_ua

desired_capabilities = DesiredCapabilities.CHROME
desired_capabilities["pageLoadStrategy"] = "none"

import winreg

user_agents = [
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36",
    "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36",
    "Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36",
    "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36",
    "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36",
    "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36",
    "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36",
    "Mozilla/5.0 (Windows NT 6.3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36",
    "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36",
    "Mozilla/5.0 (Windows NT 11.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36",
]


def generate_random_ua():
    return random.choice(user_agents)



class MessageSender(QObject):
    textUpdateSignal = pyqtSignal(str)
    progressUpdateSignal = pyqtSignal(int)
    topprogressUpdateSignal = pyqtSignal(int)
    button_1UpdateSignal = pyqtSignal(bool)

    # button_asinUpdateSignal = pyqtSignal(bool)
    button_linkUpdateSignal = pyqtSignal(bool)


class TCrawl(QWidget):

    def __init__(self, userdata):
        super().__init__()
        self.pagetodb = 1
        #读取完3页，写数据库；
        self.useproxy = userdata['useproxy']
        #  每次显示进度条，每个时间间隔多少，比如如下为0.055,则间隔为random.random()*0.055
        self.progressbarsleep = 0.055
        self.processtasktimes = 0
        #执行任务次数
        self.button_1 = QPushButton('图片抓取亚马逊竞品')
        self.button_1.clicked.connect(lambda: self.parse_next_url(0))
        cur1 = os.getcwd()
        cur1 = os.getcwd()
        cur1 = os.getcwd()

        # 这个2个程序实现共用代码，
        self.labelasin = QLabel(f'多个asin之间用空格 分割')
        self.labellink = QLabel(f'多个链接或者关键词之间用,分割，链接请不要填写asin的链接！')

        self.textasin = QLineEdit()
        # textstr = f'B08GFYTZGH B0BPGV2XBP'
        # self.textasin.setText(textstr)

        self.textlink = QLineEdit()
        # self.textlink.setText(
        #     'https://www.amazon.com/s?k=tree+light&i=tools&ref=nb_sb_noss'
        # )

        self.labeltoplink = QLineEdit(
            f'请输入顶级link，比如家居类目的顶级链接，格式参考https://www.amazon.com/home-garden-kitchen-furniture-bedding/b/?ie=UTF8&node=1055398&ref_=nav_cs_home'
        )
        self.texttoplink = QLineEdit()
        self.texttoplink.setText(
            'https://www.amazon.com/home-garden-kitchen-furniture-bedding/b/?ie=UTF8&node=1055398&ref_=nav_cs_home'
        )

        self.button_toplink = QPushButton('顶级链接抓取亚马逊竞品')
        self.button_toplink.clicked.connect(lambda: self.parse_next_url(3))

        # self.button_asin = QPushButton('多asin抓取亚马逊竞品')
        # self.button_asin.clicked.connect(lambda: self.parse_next_url(1))

        self.button_link = QPushButton('链接抓取速卖通竞品')
        self.button_link.clicked.connect(lambda: self.parse_next_url(2))

        self.textkeyword = QLineEdit()
        self.textkeyword.setText('tree light,tree light indoor')

        self.button_keyword = QPushButton('关键词抓取亚马逊竞品')
        self.button_keyword.clicked.connect(lambda: self.parse_next_url(4))

        self.scraper_thread = None
        self.topscraper_thread = None

        self.driver = None
        self.random_ua = random.choice(user_agents)
        self.account_info = getaccountinfo()
        self.account_info[
            'proxyapi'] = 'http://api.ip.data5u.com/dynamic/get.html?order=a0aab4a6ac5b39318e5317e8800a119d&random=2&sep=3'
        self.engine = create_engine(
            f"mysql+pymysql://root:Zb_200407@gz-cdb-pr3j7lhn.sql.tencentcdb.com:26590/amazon"
        )
        self.dbconn = dbhelper(server='gz-cdb-pr3j7lhn.sql.tencentcdb.com',
                               uid='root',
                               pwd='Zb_200407',
                               database='amazon',
                               port=26590,
                               creator=pymysql)

        self.connection = self.engine.connect()
        menubar = QMenuBar(self)
        file_menu = QMenu('文件', self)
        help_action = QAction('帮助', self)
        close_action = QAction('关闭', self)
        file_menu.addAction(help_action)
        file_menu.addAction(close_action)
        menubar.addMenu(file_menu)
        help_action.triggered.connect(self.showhelp)
        close_action.triggered.connect(self.closewindow)
        self.userdata = userdata
        self.username = ''
        if 'username' in self.userdata and self.userdata['username'] != '':
            self.username = userdata["username"]
        self.accountinfo = getaccountinfo()
        self.accountinfo[
            'proxyapi'] = 'http://api.ip.data5u.com/dynamic/get.html?order=a0aab4a6ac5b39318e5317e8800a119d&random=2&sep=3'
        account_info = self.accountinfo
        self.brandinfo = getbrandinfo()
        self.engine = create_engine(
            f"mysql+pymysql://root:Zb_200407@gz-cdb-pr3j7lhn.sql.tencentcdb.com:26590/amazon"
        )
        self.dbconn = dbhelper(server='gz-cdb-pr3j7lhn.sql.tencentcdb.com',
                               uid='root',
                               pwd='Zb_200407',
                               database='amazon',
                               port=26590,
                               creator=pymysql)

        self.connection = self.engine.connect()
        # self.table_combo = QComboBox()
        # self.table_combo.addItems(
        #     ['待处理表asinlist', '处理结果表asinexisting', '处理完毕表asinlist_processed'])
        w_width = 1280
        w_height = 720
        desktop = QDesktopWidget()
        screen_rect = desktop.screenGeometry()
        percentage = 85
        width = int(screen_rect.width() * percentage / 100)
        height = int(screen_rect.height() * percentage / 100)
        self.resize(width, height)
        current_path = os.getcwd()
        intclientuse = 1  #是否客户使用
        self.setWindowTitle(f"选品系统->当前用户:{self.username}")
        self.topprogress_bar = QProgressBar()

        self.progress_bar = QProgressBar()

        self.pai_text = QLabel(
            '如有问题请联系作者：492524133@qq.com。 温馨提示：此软件只提供学习交流 \n不得用于其他非法用途， 使用本软件所造成的一切后果\n皆有使用者自主承担'
        )
        self.entry_data = QLabel(f'系统运行的目录：{current_path}')
        #界面只保留登录界面，其他界面都是从数据库下载后执行；需要有一个版本控制系统
        """创建label_user按钮，与说明书"""
        """创建label_user关联输入"""
        # self.file_lable = QLabel('处理的文件名：')
        # self.entry_filepath = QLineEdit()
        # self.entry_filepath.setText('预处理ASIN数据后，这里显示导入的文件名。预处理的文件，第一行需要删除')
        self.textmemo = QTextEdit()
        self.sender = MessageSender()
        self.sender.textUpdateSignal.connect(self.textmemo.append)

        self.sender.progressUpdateSignal.connect(self.progress_bar.setValue)
        self.sender.topprogressUpdateSignal.connect(
            self.topprogress_bar.setValue)

        self.sender.button_1UpdateSignal.connect(self.button_1.setEnabled)
        # self.sender.button_asinUpdateSignal.connect(
        #     self.button_asin.setEnabled)
        self.sender.button_linkUpdateSignal.connect(
            self.button_link.setEnabled)

        self.button_2 = QPushButton('清除文本内容')
        self.button_2.clicked.connect(self.clearmemo)
        if self.username == None or self.username == '':
            self.textmemo.append('没有正常登录，无法使用该功能！')
            time.sleep(1)
            # self.close()
        str = f'需要定制程序，在定制程序之前，需要看手册\n'
        self.textmemo.append(str)
        for key, value in self.accountinfo.items():
            self.textmemo.append(f'配置参数：Key: {key}->=Value: {value}')
        pre_layout = QHBoxLayout()
        pre_layout1 = QHBoxLayout()

        top_layout = QHBoxLayout()

        top_layout.addWidget(self.button_1)
        # top_layout.addWidget(self.button_3)
        top_layout.addWidget(self.button_2)

        pre_layout1.addWidget(self.pai_text)
        pre_layout.addWidget(self.entry_data)
        main_layout = QVBoxLayout()
        main_layout.addWidget(menubar)  # 前置操作
        main_layout.addLayout(pre_layout1)  # 前置操作
        main_layout.addLayout(pre_layout)  # 前置操作
        main_layout.addLayout(top_layout)
        top_layout2 = QHBoxLayout()

        top_layout21 = QHBoxLayout()

        top_layout21.addWidget(self.texttoplink)
        top_layout21.addWidget(self.button_toplink)
        top_layout21.addWidget(self.labeltoplink)

        top_layout3 = QHBoxLayout()

        top_layout3.addWidget(self.labelasin)
        top_layout3.addWidget(self.textasin)
        # top_layout3.addWidget(self.button_asin)
        top_layout3.addWidget(self.labellink)
        self.combo_box = QComboBox()
        model = QStandardItemModel()
        self.combo_box.setModel(model)
        self.combo_box.addItems(["1：从输入框读取连接", "2：从弹出文件读取,文件里，不同的连接用换行分割"])
        self.combo_box.setCurrentIndex(0)  # 设置默认选项为第一个（索引为 0）

        top_layout3.addWidget(self.combo_box)
        top_layout3.addWidget(self.textlink)

        top_layout3.addWidget(self.button_link)
        top_layout3.addWidget(self.textkeyword)

        top_layout3.addWidget(self.button_keyword)

        main_layout.addLayout(top_layout2)
        main_layout.addLayout(top_layout21)

        main_layout.addLayout(top_layout3)
        main_layout.addWidget(self.topprogress_bar)

        main_layout.addWidget(self.progress_bar)
        main_layout.addWidget(self.textmemo)
        # main_layout.addWidget(self.table_view)
        userdata['action'] = 'choosegoods'
        userdata['memo'] = '登录选品界面'
        if not (self.username == None or self.username == ''):
            write_log(self.userdata['dbconnection'], self.userdata, None)
        self.setLayout(main_layout)

    def on_return_pressed(self):
        print("回车被按下，触发了方法。")
        self.query_data()

    def closewindow(self):
        self.close()

    def showhelp(self):
        a = 1
        sql = 'select * from us_config where key1="help"'
        dbo = self.userdata['dbconnection']
        rows = dbo.dbread_data(sql)
        self.textmemo.append(
            '\n\n<span style="color=red">请认真看下面的帮助信息哦！\n\n</span>')
        intx = 0
        for ro in rows:
            intx += 1
            styled_text = f"<span style='color:green;font-weight:bold;font-size:16pt'>{ro[1]}</span>"
            if intx % 2 == 0:
                styled_text = f"<span style='color:blue;font-weight:bold;font-size:16pt'>{ro[1]}</span>"
            self.textmemo.append(styled_text)

        self.textmemo.verticalScrollBar().setValue(
            self.textmemo.verticalScrollBar().maximum())

    def delspecchar(self, str):
        return str
        pass

    def scroll_to_random_position(self, driver):
        # 获取页面的高度
        total_height = driver.execute_script(
            "return document.body.scrollHeight")
        random_position = random.randint(int(total_height * 0.7), total_height)
        driver.execute_script(f"window.scrollTo(0, {random_position});")

    def clearmemo(self):
        """定义触发事件2，删除文本框中内容"""
        self.textmemo.setText("")
        self.userdata['action'] = 'choosegoods:cleartext'
        self.userdata['memo'] = '清除文本'
        if self.userdata['username']:
            write_log(self.userdata['dbconnection'], self.userdata, None)

    def parse_next_url(self, param):

        # self.button_1.setEnabled(False)
        # self.button_asin.setEnabled(False)
        # self.button_link.setEnabled(False)

        #parseip(self)
        account_info = self.account_info
        intuseself = 1  #0代表给客户使用，1表示自己使用

        now = datetime.datetime.now()
        current_time = now.strftime("%Y-%m-%d %H:%M:%S")
        for k in range(5):
            logging.info('')

        logging.info(
            f'开始记录日志时间：{current_time}，技术支持：492524133@qq.com,加vx:yiliyamei注明sif分析.谢谢'
        )
        headers = {"User-Agent": self.random_ua}
        current_path = os.getcwd()

        ver1 = get_chrome_version()
        if intuseself == 1:
            self.textmemo.append(f'GOOGLE浏览器版本{ver1}文件\n')
        dictsave = {}  #保存处理后的数据字典
        asindata = []
        driver = None
        try:
            chrome_options = Options()
            chrome_options.add_argument(f"user-agent={headers['User-Agent']}")
            chrome_options.add_argument('--disable-dev-shm-usage')
            if intuseself == 0:
                chrome_options.add_argument("--headless")  # 添加无头模式参数
            chrome_options.add_argument("--no-sandbox")
            chrome_options.add_experimental_option('excludeSwitches',
                                                   ['enable-automation'])
            chrome_options.add_argument('--ignore-certificate-errors')
            chrome_options.add_argument('--ignore-ssl-errors')
            # 创建一个webdriver实例
            chrome_options.add_argument('--disable-gpu-vsync')

            # proxy=self.getproxyip()
            # if proxy!='' and has_ip_address(proxy) :
            #     chrome_options.add_argument(f'--proxy-server={proxy}')

            # chrome_options.add_argument('--no-proxy-server')
            # proxy1=self.getproxyip()
            # chrome_options.add_argument(f'--proxy-server={proxy1}')
            current_path = os.getcwd()
            # 设置代理
            #proxy2=ipdatalist[random.randint(0,len(ipdatalist)-1)]
            #chrome_options.add_argument(f'--proxy-server={proxy2}')
            chromedriver_path = os.path.join(current_path, "chromedriver.exe")
            logging.info(f'chromedriver_path={chromedriver_path}')

            # 创建 Service 对象并传入路径
            service = Service(executable_path=chromedriver_path)

            # 创建 WebDriver 实例时传入 Service 对象和 ChromeOptions
            driver = webdriver.Chrome(service=service, options=chrome_options)
            ver2 = driver.capabilities['chrome']['chromedriverVersion'].split(
                ' ')[0]
            if intuseself == 1:
                self.textmemo.append(f'chromedriver版本{ver2}\n')
            logging.info(f'chromedriver版本{ver2}\n')
            driver.implicitly_wait(3)
        except Exception as e:
            #打开浏览器报错
            print(e)
            logging.info(f'访问浏览器，出现错误：{e}\n')
            # self.textmemo.append( f'访问sif.comasin：{column_a_value}出现错误：{e}\n')
            self.textmemo.verticalScrollBar().setValue(
                self.textmemo.verticalScrollBar().maximum())
        self.textmemo.verticalScrollBar().setValue(
            self.textmemo.verticalScrollBar().maximum())
        self.reset_proxyip(driver)
        driver.get("https://www.ebay.com")
        wait = WebDriverWait(driver, 10)
        wait.until(lambda driver: driver.execute_script(
            'return document.readyState') == 'complete')
        driver.execute_script(
            "window.scrollTo(0, document.body.scrollHeight);")
        time.sleep(random.random() / 4)
        # 将平行滚动条拉到最右侧
        driver.execute_script("window.scrollTo(document.body.scrollWidth, 0);")
        textdd = "这是第一次访问sif关网页，如果你准备好了!,请点击确认"

        msg_box = QMessageBox()
        msg_box.setIcon(QMessageBox.Question)
        msg_box.setText("将亚马逊网站，设置为英文版！！如果你准备好了网页,请确认！")
        msg_box.setWindowTitle("在抓取过程中不要关闭打开的浏览器，否则出错")
        msg_box.setStandardButtons(QMessageBox.Yes | QMessageBox.No)
        response = msg_box.exec_()
        if response == QMessageBox.Yes:
            print("User is ready.")
        else:
            print("User is not ready.")
        self.driver = driver
        strparams = ''
        xp1 = '//*[@id="tabProdoct"]'
        driver.find_element(By.XPATH, '//*[@id="tabProdoct"]').click()

driver.find_element(By.XPATH, '//*[@id="tabProdoct"]/div/div[1]/div[1]/div/div[2]/div[1]/a').click()
        
        # input.click()
        self.reset_proxyip(driver=driver)
        if not self.scraper_thread or not self.scraper_thread.is_alive():

            if param == 4:
                strtextlinks = ''
                strlink = ''
                strkeyword = self.textkeyword.text()
                for k1 in strkeyword.split(','):
                    strlink = ''
                    for j1 in k1.split(' '):
                        strlink = strlink + j1.strip() + '+'
                    if len(strlink) > 0:
                        strlink = strlink[:-1]
                    strlink = f"https://www.aliexpress.com/s?k={strlink}&i=tools&ref=nb_sb_noss"
                    strtextlinks = strtextlinks + strlink + ','
                if len(strtextlinks) > 0:
                    strtextlinks = strtextlinks[:-1]
                self.textlink.setText(strtextlinks)
                self.scraper_thread = threading.Thread(
                    target=self.long_running_task,
                    args=(2, self.textlink.text()))
            if param == 3:
                # 解析2次
                self.topscraper_thread = threading.Thread(
                    target=self.long_running_task,
                    args=(param, self.texttoplink.text()))
            if param == 0:
                self.scraper_thread = threading.Thread(
                    target=self.long_running_task, args=(param, ))

            print("Before starting thread.")
            if param == 3:
                self.topscraper_thread.start()
            else:
                self.scraper_thread.start()
            print("After starting thread.")
            # self.button_1.setEnabled(False)
    def reset_proxyip(self, driver):
        proxy = self.getproxyip()
        if proxy != '' and has_ip_address(proxy):
            proxy_1 = proxy.split(':')
            driver.proxy = {
                'http': f'http://{proxy_1[0]}:{proxy_1[1]}',
                'https': f'http://{proxy_1[0]}:{proxy_1[1]}'
            }

    def getproxyip(self):
        proxyip = 'no proxy'
        try:
            apiUrl = self.accountinfo['proxyapi']
            res = requests.get(apiUrl).content.decode()
            # 按照\n分割获取到的IP
            ips = res.split('\n')
            # 利用每一个IP
            for proxyip2 in ips:

                if proxyip2.strip() == '':
                    continue
                else:
                    proxyip = proxyip2
                print(proxyip2)
                self.textmemo.append(f'使用代理ip{proxyip}访问')
        except Exception as e:
            print(f'getproxyip{e}')
        return proxyip

    def long_running_task(self, param1=0, param2=None):

        self.processtasktimes += 1
        if not 'username' in self.userdata or self.userdata['username'] == '':
            self.sender.textUpdateSignal.emit(f"请尽快登录哦")

        if self.processtasktimes > 5 and (not 'username' in self.userdata
                                          and self.userdata['username'] == ''):
            self.sender.textUpdateSignal.emit(
                f"已经执行{self.processtasktimes}次，请去登录或者注册后登录")
            return
        print(f"long_running_task{param1} starting thread.")
        result = None
        driver = self.driver
        products = []
        if param1 == 0:
            body_element = self.driver.find_element(By.XPATH, "/html/body")
            body_element.click()

            elements = self.driver.find_elements(By.CLASS_NAME,
                                                 'cellContainer')
            total_elements = len(elements)
            index = 0
            sourceasin = ''
            for element in elements:
                try:
                    soup = BeautifulSoup(element.get_attribute("outerHTML"),
                                         'html.parser')
                    # 现在可以使用 BeautifulSoup 的方法来处理这个元素
                    asin = soup.find('article')['data-csa-c-item-id'].split(
                        '.')[-1]
                    if not sourceasin or sourceasin == '':
                        sourceasin = asin
                    price = soup.find('span').text.replace('US$', '')
                    rating = soup.find('p', class_='value').text
                    brandname = soup.find('h5').text  #品牌名
                    review_count = soup.find('p', class_='count').text.replace(
                        ',', '')  #评价数
                    link = f'https://www.aliexpress.com/dp/{asin}'
                    print(
                        f"ASIN: {asin}, Price: {price}, Rating: {rating}, Product Name: {brandname}, Review Count: {review_count}"
                    )
                    product_dict = {
                        'sourceasin':
                        sourceasin,
                        'asin':
                        asin,
                        'price':
                        price,
                        'rating':
                        rating,
                        'review_count':
                        review_count,
                        'username':
                        self.userdata['username'],
                        'opdate':
                        datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
                        'brandname':
                        brandname,
                        'link':
                        link,
                        'piclink':
                        '',
                        'passbuy':
                        '0'
                    }
                    products.append(product_dict)
                    # self.textmemo.append(f"数据下载正确: {asin}")
                    # self.progress_bar.setValue(
                    #     int((index + 1) * 100 / total_elements))
                    self.sender.textUpdateSignal.emit(
                        f"第{index+1}行,asin：{asin}数据下载正确 ")
                    self.sender.progressUpdateSignal.emit(
                        int((index + 1) * 100 / total_elements))
                    index += 1
                    time.sleep(random.random() * self.progressbarsleep)
                except Exception as e:
                    print(e)
                    # self.textmemo.append(f"数据下载失败: {asin},错误信息{e}")
                    self.sender.textUpdateSignal.emit(
                        f"数据下载失败: {asin},错误信息{e}")
                    # self.textmemo.append(e)
                # self.textmemo.append("抓取完成。")

        if param1 == 2:
            actions = ActionChains(self.driver)
            links = param2.split(',')
            links.append('testok')
            index = 0
            intpage = 0
            for link in links:
                key1 = ''
                products = []
                index += 1
                match = re.search(r'k=(.*)', link)
                if match:
                    key1 = match.group(1)
                key1 = key1.split('&')[0].replace('+', ' ')
                intpage += 1
                if index == len(links):
                    continue
                self.sender.textUpdateSignal.emit(f"准备处理链接{link} ")
                self.reset_proxyip(driver=driver)
                # self.driver.get(link)
                driver_get(self, link)
                body_element = self.driver.find_element(By.XPATH, "/html/body")
                body_element.click()
                document_height = self.driver.execute_script(
                    "return document.body.scrollHeight")
                # 生成一个随机的滚动位置
                random_scroll_position = random.randint(
                    0, int(document_height / 2))
                self.driver.execute_script(
                    f"window.scrollTo(0, {random_scroll_position});")
                time.sleep(random.random() * 1.2)
                random_scroll_position2 = random.randint(
                    int(document_height / 2), document_height)
                self.driver.execute_script(
                    f"window.scrollTo({random_scroll_position}, {random_scroll_position2});"
                )
                ifnext = 1
                time.sleep(random.random())
                self.driver.execute_script(
                    f"window.scrollTo({random_scroll_position2}, {document_height});"
                )
                intinnerpage = 0
                while (ifnext):
                    intinnerpage += 1

                    manscroll(self.driver)
                    wait = WebDriverWait(self.driver, 10)

                    elements = driver.find_elements(
                        By.CSS_SELECTOR,
                        '[data-component-type="s-search-result"]')
                    indexj = 0
                    for element in elements:
                        indexj += 1
                        asin = ''
                        try:
                            soup = BeautifulSoup(
                                element.get_attribute("outerHTML"),
                                'html.parser')
                            # 现在可以使用 BeautifulSoup 的方法来处理这个元素

                            asin = soup.find(
                                attrs={'data-asin': True
                                       })['data-asin'] if soup.find(
                                           attrs={'data-asin': True}) else None
                            aaa = 3333

                            if asin == 'B0D6R573KX':
                                aaa = 333

                            price = '0'
                            rating = '0'
                            ratings_count = '0'
                            try:
                                price = soup.find(
                                    'span',
                                    class_='a-offscreen').text if soup.find(
                                        'span', class_='a-offscreen') else None
                                if price:
                                    price = price.replace('US$', '')
                                if price and not is_floatstr(price):
                                    price = '0'
                                if not price:
                                    price = '0'
                                try:
                                    rating = soup.find(
                                        'span', {
                                            'aria-label':
                                            lambda value: value and
                                            value.split()[0].replace(
                                                '.', '', 1).isdigit()
                                        }).text.split()[0] if soup.find(
                                            'span', {
                                                'aria-label':
                                                lambda value: value and value
                                                .split()[0].replace(
                                                    '.', '', 1).isdigit()
                                            }) else None

                                    ratings_count = soup.find(
                                        'span',
                                        class_='a-size-base s-underline-text'
                                    ).text.replace(',', '').replace(' ', '')

                                except Exception as e:
                                    print(e)
                                    if not rating:
                                        rating = soup.find_all(
                                            'span', attrs={
                                                'aria-label': True
                                            })[0].text.split(' ')[0]  #评分比如4.2
                                    if not ratings_count:
                                        ratings_count = soup.find_all(
                                            'span', attrs={
                                                'aria-label': True
                                            })[1].text.split(' ')[0].replace(
                                                ',', '')  #评价数量比如1301

                                finally:
                                    if not rating:
                                        rating = '0'
                                    if not ratings_count:
                                        ratings_count = '0'
                                    if rating and not is_floatstr(rating):
                                        rating = '0'
                                    if ratings_count and not is_floatstr(
                                            ratings_count):
                                        ratings_count = '0'
                            except Exception as e:
                                print(e)
                            print(asin)
                            print(price)
                            print(rating)
                            print(ratings_count)
                            image_container = soup.find(
                                'div', class_='s-product-image-container')
                            piclink = ''
                            if image_container:
                                img_tags = image_container.find_all(
                                    'img', {'class': 's-image'})
                                jpg_links = [
                                    img['src'] for img in img_tags
                                    if img['src'].endswith('.jpg')
                                ]
                                for linkx in jpg_links:
                                    piclink = linkx
                                    print(linkx)
                            else:
                                print("未找到图片容器元素")
                            # 过去一个月有多少人买
                            passbuy = '0+'
                            buytext_element = soup.find(
                                'span', class_='a-size-base a-color-secondary')
                            if buytext_element:
                                passbuy = buytext_element.text
                                print(passbuy)
                            else:
                                print("未找到对应元素")
                            if passbuy != None:
                                passbuy = passbuy.lower().replace('k', '000')
                                match = re.search(r'\d+\+', passbuy)
                                if match:
                                    passbuy = match.group()
                                else:
                                    passbuy = '0+'
                                passbuy = passbuy.lower().replace('+', '')
                            else:
                                passbuy = '0'
                            try:
                                whole1 = soup.find('span',
                                                   class_='a-price-whole').text
                                fraction1 = soup.find(
                                    'span', class_='a-price-fraction').text
                                price = whole1 + fraction1
                            except Exception as e:
                                print(e)
                            product_dict = {
                                'sourceasin':
                                key1,
                                'asin':
                                asin,
                                'price':
                                price,
                                'rating':
                                rating,
                                'review_count':
                                ratings_count,
                                'username':
                                self.userdata['username'],
                                'opdate':
                                datetime.datetime.now().strftime(
                                    '%Y-%m-%d %H:%M:%S'),
                                'brandname':
                                '',
                                'link':
                                f"https://www.amazon.com/dp/{asin}",
                                'piclink':
                                piclink,
                                'passbuy':
                                passbuy
                            }

                            products.append(product_dict)
                            time.sleep(random.random() *
                                       self.progressbarsleep * 0.1)
                            intpro = int(indexj * 100 / len(elements))
                            if intpro > 100:
                                intpro = 100
                            self.sender.progressUpdateSignal.emit(intpro)
                            self.sender.textUpdateSignal.emit(
                                f"第{intpage}个链接，第{intinnerpage}页,第{indexj}行asin:{product_dict}下载正常！ "
                            )
                            if intinnerpage % self.pagetodb == 0:
                                thread2 = threading.Thread(
                                    target=self.asin2amz, args=(products, ))
                                products = []
                                thread2.start()

                        except Exception as e:
                            self.sender.textUpdateSignal.emit(
                                f"asin:{asin}:出现异常:{e} ")

                    try:

                        page_width = driver.execute_script(
                            "return document.body.scrollWidth")

                        # 滚动到最右侧
                        driver.execute_script(
                            f"window.scrollTo({page_width}, 0);")
                        page_source2 = driver.page_source
                        # 使用 BeautifulSoup 解析页面源代码
                        soup2 = BeautifulSoup(page_source2, 'html.parser')
                        link_element = soup2.find(
                            'a',
                            class_=
                            's-pagination-item s-pagination-next s-pagination-button s-pagination-separator'
                        )
                        span_element = soup2.find(
                            'span',
                            class_=
                            's-pagination-item s-pagination-next s-pagination-disabled'
                        )
                        if link_element:
                            aria_label = link_element['aria-label']
                            href = "https://www.amazon.com" + link_element[
                                'href']
                            print(aria_label)
                            print(href)
                            ifnext = 1
                        else:
                            print("Link not found.")
                            ifnext = 0
                        is_disabled = False
                        if span_element:
                            is_disabled = span_element.has_attr(
                                'aria-disabled'
                            ) and span_element[
                                'aria-disabled'] == "true" if span_element else False

                        if is_disabled:
                            print("Next link is disabled and not valid.")
                            ifnext = 0
                        else:
                            print("Next link is valid.")
                            ifnext = 1
                        if ifnext == 1:
                            # self.driver.get(href)

                            driver_get(self, href)
                        # next_link = driver.find_element_by_css_selector('a.s-pagination-item.s-pagination-next.s-pagination-button.s-pagination-separator')
                        # 获取链接地址
                    except Exception as e:
                        print(e)
                        ifnext = 0
                # self.asin2amz(products)
                #大约延迟一秒；
        if param1 == 3:
            # actions = ActionChains(self.driver)
            links2 = param2.split(',')
            links2.append('testok')
            index = 0
            intpage = 0
            # 第一层循环
            for link2 in links2:
                products = []
                index += 1
                intpage += 1
                if index == len(links2):
                    continue
                self.sender.textUpdateSignal.emit(f"准备分析链接{link2} ")
                self.reset_proxyip(driver=driver)
                # driver_get(self,link2)
                self.driver.get(link2)
                time.sleep(1)
                document_height = self.driver.execute_script(
                    "return document.body.scrollHeight")
                # 生成一个随机的滚动位置
                random_scroll_position = random.randint(
                    0, int(document_height / 2))
                self.driver.execute_script(
                    f"window.scrollTo(0, {random_scroll_position});")
                time.sleep(random.random() * 1.2)
                random_scroll_position2 = random.randint(
                    int(document_height / 2), document_height)
                self.driver.execute_script(
                    f"window.scrollTo({random_scroll_position}, {random_scroll_position2});"
                )
                time.sleep(random.random())
                self.driver.execute_script(
                    f"window.scrollTo({random_scroll_position2}, {document_height});"
                )
                try:
                    wait = WebDriverWait(self.driver, 10)
                    elements1 = wait.until(
                        EC.presence_of_all_elements_located(
                            (By.CSS_SELECTOR,
                             'bxc-grid__image.bxc-grid__image--light')))
                    indexj = 0
                    for element1 in elements1:
                        asin = ''
                        intpro = int(indexj * 100 / len(elements1))
                        if intpro > 100:
                            intpro = 100
                        self.sender.topprogressUpdateSignal.emit(intpro)
                        indexj += 1

                        soup1 = BeautifulSoup(
                            element1.get_attribute("outerHTML"), 'html.parser')
                        # 现在可以使用 BeautifulSoup 的方法来处理这个元素
                        # 获取链接
                        li_element = soup1.find(
                            'div',
                            class_=
                            'bxc-grid__column  bxc-grid__column--2-of-12   bxc-grid__column--light'
                        )
                        href_value = ''
                        a_tag = soup1.find('a', {'aria-label': 'Furniture'})
                        if a_tag:
                            href_value = a_tag.get('href')
                            print(href_value)
                        else:
                            print("未找到指定的 <a> 标签")
                        self.reset_proxyip(driver=driver)
                        # driver_get(self,href_value)
                        self.driver.get(href_value)
                        wait = WebDriverWait(self.driver, 10)
                        elements2 = wait.until(
                            EC.presence_of_all_elements_located(
                                (By.CSS_SELECTOR,
                                 'a-carousel-card dcl-carousel-element')))
                        for element2 in elements2:
                            soup2 = BeautifulSoup(
                                element1.get_attribute("outerHTML"),
                                'html.parser')
                            # 现在可以使用 BeautifulSoup 的方法来处理这个元素
                            # 获取链接
                            if li_element2:
                                a_tag = li_element2.find(
                                    'a',
                                    class_='a-color-base a-link-normal a-block'
                                )
                                if a_tag:
                                    href_value2 = a_tag.get('href')
                                    print(href_value2)
                                    self.sender.textUpdateSignal.emit(
                                        f"准备分析链接{href_value2} ")

                                    self.scraper_thread = threading.Thread(
                                        target=self.long_running_task,
                                        args=(2, href_value2))
                                    self.scraper_thread.start()
                            else:
                                print("未找到目标列表项元素")
                except Exception as e:
                    print(e)
                    self.sender.textUpdateSignal.emit(f"出现错误{e} ")

        if param1 != 2:
            self.asin2amz(products)

        self.sender.button_1UpdateSignal.emit(True)
        # self.sender.button_asinUpdateSignal.emit(True)
        self.sender.button_linkUpdateSignal.emit(True)

        self.driver.quit()

    def asin2amz(self, products):
        engine = self.engine
        Session = sessionmaker(bind=self.engine)
        session = Session()
        placeholders = ', '.join(['(%s, %s, %s, %s,%s, now(), %s, %s)'] *
                                 len(products))
        values = []
        index = 0
        total_elements = len(products)
        # sourceasin = ''
        # if len(products) > 1:
        #     sourceasin = products[0]['asin']
        # elif len(products) == 1:
        #     sourceasin = products['asin']
        for product in products:
            asin = product['asin']
            sql = f"""
            INSERT INTO asinlist (sourceasin,asin, 价格, 评分, 评分数, 品牌, username, 商品详情页链接,opdate,商品主图,passmonthbuy)
            VALUES (:sourceasin,:asin, :price, :rating, :review_count, :brandname, :username, :link,{self.userdata['datefunction']},:piclink,:passbuy);
            """

            if product['price']:
                product['price'] = product['price'].replace('US$', '')
            elif not product['price']:
                product['price'] = '0'

            if product['rating'] and not is_floatstr(product['rating']):
                product['rating'] = '0'
            elif not product['rating']:
                product['rating'] = '0'

            if product['review_count'] and not is_floatstr(
                    product['review_count']):
                product['review_count'] = '0'
            elif not product['review_count']:
                product['review_count'] = '0'
            try:
                with engine.connect() as connection:
                    values = tuple(product.values())
                    connection.execute(text(sql), product)
                    connection.commit()
                self.sender.textUpdateSignal.emit(
                    f"第{index+1}行,asin：{asin}数据插入数据库{product}正确")
                # self.textmemo.append(f"数据库插入正确: {asin}")
            except Exception as e:
                print(e)
                self.sender.textUpdateSignal.emit(f"数据库插入错误: {asin},错误信息{e}")
                # self.textmemo.append(f"数据库插入错误: {e}")
            # self.progress_bar.setValue(int((index + 1) * 100 / total_elements))
            self.sender.progressUpdateSignal.emit(
                int((index + 1) * 100 / total_elements))
            index += 1
            time.sleep(random.random() * self.progressbarsleep)
        # 执行插入操作
        self.sender.textUpdateSignal.emit(f"请去选品系统去研究sif数据")
        # self.textmemo.append(
        #     "抓取数据写入us_products完成，请出查询或者导出，然后开产品定位广告，定位到不如你的产品所有动作完成。")
        if session:
            session.close()


def source_path(relative_path):
    if getattr(sys, 'frozen', False):
        base_path = sys._MEIPASS
    else:
        base_path = os.path.abspath(".")
    return os.path.join(base_path, relative_path)


if __name__ == '__main__':
    try:
        accountinfo = getaccountinfo()
        databasetype = accountinfo['databasetype']
        dbconn = None

        if databasetype.lower() != 'mysql':
            print('a23c')
            self.dbconn = dbhelper()
            print('a23d')
            datefunction = 'getdate()'
        else:
            datefunction = 'now()'
            print('a23')
            dbconn = dbhelper(server='gz-cdb-pr3j7lhn.sql.tencentcdb.com',
                              uid='root',
                              pwd='Zb_200407',
                              database='amazon',
                              port=26590,
                              creator=pymysql)
        userdata = {
            'gateway': '',
            'ip': 'self.ip',
            'username': 'q11',
            'disknumber': 'self.disknumber',
            'dbconnection': dbconn,
            'databasetype': databasetype,
            'datefunction': 'now()',
            'debug': 1,
            'useproxy':1
        }
        # userdata = {
        #     'gateway': '',
        #     'ip': 'self.ip',
        #     'disknumber': 'self.disknumber',
        #     'dbconnection': dbconn,
        #     'databasetype': databasetype,
        #     'datefunction': 'now()',
        #     'debug': 1
        # }
        cd = source_path('')
        os.chdir(cd)
        # userdata = {'username': 'q11'}
        app = QApplication(sys.argv)
        ts = TCrawl(userdata)
        ts.show()
        app.exec_()
    except Exception as e:
        print(e)
